package me.chendan

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by chendan on 16-4-4.
  */
object WordCount {
  def main(args: Array[String]) {
    val filePath = args.length match {
      case x: Int if (x > 0) => args(0)
      case _ => "README.md"
    }
    //val filePath = "/home/chendan/derby.log"
    val conf = new SparkConf().setMaster("local[2]").setAppName("wordCount")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(filePath)
    val wordMap = logData.flatMap(line => line.split(" ")).map(w => (w, 1))
    val result = wordMap.reduceByKey((a, b) => a + b)
    result.foreach(println)
    println("finished")
  }
}
